{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "import html2text\n",
    "from urllib.parse import unquote, quote\n",
    "\n",
    "\n",
    "from pprint import pprint\n",
    "from alive_progress import alive_bar\n",
    "from attributesDir.attributesDir import  adir, adoc, aprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib.parse as p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ParseResult(scheme='https', netloc='hellogithub.com', path='/periodical/category/C%2B%2B%20%E9%A1%B9%E7%9B%AE/', params='', query='page=1', fragment='')"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p.urlparse(\"https://hellogithub.com/periodical/category/C%2B%2B%20%E9%A1%B9%E7%9B%AE/?page=1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## options"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_page_url = scheme + \"://\" + netloc\n",
    "\n",
    "main_page_html = requests.get(main_page_url).text\n",
    "\n",
    "\n",
    "soup = BeautifulSoup(main_page_html, \"html.parser\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "options = soup.find_all(\"select\", attrs={\"id\":\"category\"})[0].find_all(\"option\")\n",
    "# options"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## get_total_pages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_total_pages(url_page):\n",
    "    number = 1\n",
    "    while True:\n",
    "        url = f\"{url_page}={number}\"\n",
    "        response = requests.get(url)\n",
    "        if response.status_code == 404:\n",
    "            total_pages = number - 1\n",
    "            print(f\"total pages: {total_pages}\")\n",
    "            break\n",
    "#         print(number)\n",
    "        number += 1\n",
    "    return total_pages"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## merge_markdown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_markdown(total_pages):\n",
    "    h = html2text.HTML2Text()\n",
    "    h.ignore_links = False\n",
    "    h.mark_code = True\n",
    "    markdown = str()\n",
    "\n",
    "    total_pages += 1\n",
    "    for number in range(1, total_pages):\n",
    "        url = f\"{url_page}={number}\"\n",
    "        response = requests.get(url)\n",
    "        if response.status_code == 200:\n",
    "            html = response.text\n",
    "#             print(number)\n",
    "        else:\n",
    "            break\n",
    "        text = h.handle(html)\n",
    "        start_anchor_text = \"兴趣是最好的老师，HelloGitHub 就是帮你找到编程的兴趣。\\n\\n\"\n",
    "        start_anchor_end_point = text.find(start_anchor_text) + len(start_anchor_text)\n",
    "\n",
    "        if number == 1:\n",
    "            end_anchor_text = \"\\n\\n* * *\\n\\n[下一页]\"\n",
    "        else:\n",
    "            end_anchor_text = \"\\n\\n* * *\\n\\n[上一页]\"\n",
    "        end_anchor_start_point = text.find(end_anchor_text)\n",
    "        markdown += f\"\\n## [Page {number}]({url}) \\n\" + text[start_anchor_end_point:end_anchor_start_point]\n",
    "        \n",
    "        print(f\"\\r\" + \"▇\" * number + \" \" + str(round(number/(total_pages-1) *100, 2)) + \"%\", end=\"\")\n",
    "        number += 1\n",
    "    print()\n",
    "    return markdown"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## format_markdown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_markdown(markdown):\n",
    "    markdown = markdown.replace(\"\"\"-\\n\"\"\", \"-\")\n",
    "\n",
    "    markdown = markdown.replace(\"\"\"/periodical/statistics/click/?target=\"\"\", \"\")\n",
    "\n",
    "    # markdown = markdown.replace(\"\"\"__\\n\\n\"\"\", \"\\n\\n\")\n",
    "\n",
    "    markdown = markdown.replace(\"\"\"##\\n\"\"\", \"\\n\\n\\n## \")\n",
    "\n",
    "\n",
    "    markdown = markdown.replace(\"\"\"[code]\"\"\", \"```python\")\n",
    "    markdown = markdown.replace(\"\"\"[/code]\"\"\", \"```\\n\")\n",
    "\n",
    "    markdown = re.sub(r\"\\n*\\d+ _+\\n*\",\"\", markdown)\n",
    "\n",
    "    markdown = re.sub(r\"\\*\\*_+.+ \\*\\* *\\n*\",\"   \", markdown)\n",
    "    return markdown"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## save_project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C 项目\n",
      "https://hellogithub.com/periodical/category/C%20%E9%A1%B9%E7%9B%AE/?page\n",
      "total pages: 7\n",
      "▇▇▇▇▇▇▇ 100.0%\n",
      "C# 项目\n",
      "https://hellogithub.com/periodical/category/C%23%20%E9%A1%B9%E7%9B%AE/?page\n",
      "total pages: 7\n",
      "▇▇▇▇▇▇▇ 100.0%\n",
      "C++ 项目\n",
      "https://hellogithub.com/periodical/category/C%2B%2B%20%E9%A1%B9%E7%9B%AE/?page\n",
      "total pages: 9\n",
      "▇▇▇▇▇▇▇▇▇ 100.0%\n",
      "CSS 项目\n",
      "https://hellogithub.com/periodical/category/CSS%20%E9%A1%B9%E7%9B%AE/?page\n",
      "total pages: 5\n",
      "▇▇▇▇▇ 100.0%\n",
      "Go 项目\n",
      "https://hellogithub.com/periodical/category/Go%20%E9%A1%B9%E7%9B%AE/?page\n",
      "total pages: 23\n",
      "▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 100.0%\n",
      "Java 项目\n",
      "https://hellogithub.com/periodical/category/Java%20%E9%A1%B9%E7%9B%AE/?page\n",
      "total pages: 20\n",
      "▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 100.0%\n",
      "JavaScript 项目\n",
      "https://hellogithub.com/periodical/category/JavaScript%20%E9%A1%B9%E7%9B%AE/?page\n",
      "total pages: 32\n",
      "▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 100.0%\n",
      "Kotlin 项目\n",
      "https://hellogithub.com/periodical/category/Kotlin%20%E9%A1%B9%E7%9B%AE/?page\n",
      "total pages: 2\n",
      "▇▇ 100.0%\n",
      "Objective-C 项目\n",
      "https://hellogithub.com/periodical/category/Objective-C%20%E9%A1%B9%E7%9B%AE/?page\n",
      "total pages: 4\n",
      "▇▇▇▇ 100.0%\n",
      "PHP 项目\n",
      "https://hellogithub.com/periodical/category/PHP%20%E9%A1%B9%E7%9B%AE/?page\n",
      "total pages: 4\n",
      "▇▇▇▇ 100.0%\n",
      "Python 项目\n",
      "https://hellogithub.com/periodical/category/Python%20%E9%A1%B9%E7%9B%AE/?page\n",
      "total pages: 26\n",
      "▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 100.0%\n",
      "Ruby 项目\n",
      "https://hellogithub.com/periodical/category/Ruby%20%E9%A1%B9%E7%9B%AE/?page\n",
      "total pages: 2\n",
      "▇▇ 100.0%\n",
      "Rust 项目\n",
      "https://hellogithub.com/periodical/category/Rust%20%E9%A1%B9%E7%9B%AE/?page\n",
      "total pages: 2\n",
      "▇▇ 100.0%\n",
      "Swift 项目\n",
      "https://hellogithub.com/periodical/category/Swift%20%E9%A1%B9%E7%9B%AE/?page\n",
      "total pages: 7\n",
      "▇▇▇▇▇▇▇ 100.0%\n",
      "其它\n",
      "https://hellogithub.com/periodical/category/%E5%85%B6%E5%AE%83/?page\n",
      "total pages: 36\n",
      "▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 100.0%\n",
      "开源书籍\n",
      "https://hellogithub.com/periodical/category/%E5%BC%80%E6%BA%90%E4%B9%A6%E7%B1%8D/?page\n",
      "total pages: 9\n",
      "▇▇▇▇▇▇▇▇▇ 100.0%\n",
      "机器学习\n",
      "https://hellogithub.com/periodical/category/%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0/?page\n",
      "total pages: 14\n",
      "▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 100.0%\n",
      "####################   Finish   ####################\n"
     ]
    }
   ],
   "source": [
    "for option in options:\n",
    "    value = option.attrs[\"value\"]\n",
    "    \n",
    "    text = option.text\n",
    "    project_name = text\n",
    "\n",
    "    print(project_name)\n",
    "\n",
    "    url_page = f\"https://hellogithub.com{value}?page\"\n",
    "\n",
    "    print(url_page)\n",
    "    total_pages = get_total_pages(url_page)\n",
    "    \n",
    "    markdown = merge_markdown(total_pages)\n",
    "    markdown = format_markdown(markdown)\n",
    "    file_name = f\"{project_name}.md\"\n",
    "    \n",
    "    if total_pages == 0: continue\n",
    "        \n",
    "    with open(file_name, \"w\", encoding='utf-8') as f:\n",
    "        f.write(markdown)\n",
    "print(\"#\" * 20 + \"   Finish   \" +  \"#\" * 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "757px",
    "left": "61px",
    "top": "111.125px",
    "width": "384px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
